import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.wltea.analyzer.lucene.IKAnalyzer;

import java.io.*;

public class FilterUtil {
	// 二级分类过滤词汇
	public static String[] filter = new String[]{
			"只要","我","一","成了","必须","为了","不要","了","正在","好好","的"
	};
	public static void main(String[] args) {
		String path = FilterUtil.class.getResource("").getPath();
		try (BufferedReader br = new BufferedReader(new FileReader(path+ File.separator+"atri.json"));
		     BufferedWriter bw = new BufferedWriter(new FileWriter(path+ File.separator+"atri2.json"))) {
			// 创建一个分词对象
			Analyzer analyzer = new IKAnalyzer(true);
			String row;
			int sort;
			while ((row = br.readLine()) != null) {
				StringBuilder builder = new StringBuilder();
				sort = 0;
				// 判断级别
				// 一级：所有
				if (row.contains("\"s_f\"")) sort = 1;
				// 二级：只保留两个字、三个字，简单词汇过滤
				if (row.contains("\"s_k\"")) sort = 2;
				if (sort != 0) {
					// 分词
					StringReader reader = new StringReader(row.substring(row.indexOf(":") + 3));
					// 对读入的语言开始进行分词操作
					TokenStream ts = analyzer.tokenStream("", reader);
					// 获得CharTermAttribute类
					CharTermAttribute term = ts.getAttribute(CharTermAttribute.class);
					// 依次遍历分词数据，注意要转换成字符串类型
					while (ts.incrementToken()) {
						String str = term.toString();
						if (str.length() < 2) continue;
						if (sort == 2){
							if (str.length() > 4) continue;
							boolean flag = true;
							for (String s : filter) {
								if (str.contains(s)){
									flag = false;
									break;
								}
							}
							if (!flag) continue;
						}

						if (builder.length() == 0) {
							builder.append(term);
						} else
							builder.append(" ").append(term);
					}

					// 拼接
					row = row.replace(row.substring(row.indexOf(":") + 3, row.lastIndexOf("\"")), builder);
					reader.close();
				}
				bw.write(row);
				bw.newLine();
			}
			bw.flush();
		} catch (IOException e) {
			e.printStackTrace();
		}
		System.out.println("写入完成");
	}


}
